<!DOCTYPE html>



  


<html class="theme-next gemini use-motion" lang="zh-Hans">
<head><meta name="generator" content="Hexo 3.9.0">
  <meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta name="theme-color" content="#222">



  
  
    
    
  <script src="/dxl/lib/pace/pace.min.js?v=1.0.2"></script>
  <link href="/dxl/lib/pace/pace-theme-minimal.min.css?v=1.0.2" rel="stylesheet">







<meta http-equiv="Cache-Control" content="no-transform">
<meta http-equiv="Cache-Control" content="no-siteapp">
















  
  
  <link href="/dxl/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css">







<link href="/dxl/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css">

<link href="/dxl/css/main.css?v=5.1.4" rel="stylesheet" type="text/css">


  <link rel="apple-touch-icon" sizes="180x180" href="/dxl/images/apple-touch-icon-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="32x32" href="/dxl/images/favicon-32x32-next.png?v=5.1.4">


  <link rel="icon" type="image/png" sizes="16x16" href="/dxl/images/favicon-16x16-next.png?v=5.1.4">


  <link rel="mask-icon" href="/dxl/images/logo.svg?v=5.1.4" color="#222">





  <meta name="keywords" content="爬虫,">










<meta name="description" content="一介绍1.1什么是scrapy1　Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架，非常出名，非常强悍。所谓的框架就是一个已经被集成了各种功能（高性能异步下载，队列，分布式，解析，持久化等）的具有很强通用性的项目模板  1.2安装1、linux 1pip3 install scrapy  2、windows 12345a. pip3 install wheelb. 下载twist">
<meta name="keywords" content="爬虫">
<meta property="og:type" content="article">
<meta property="og:title" content="爬虫scripy框架">
<meta property="og:url" content="http://yoursite.com/2019/09/20/【爬虫06】scrapy框架/index.html">
<meta property="og:site_name" content="我的快乐时光">
<meta property="og:description" content="一介绍1.1什么是scrapy1　Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架，非常出名，非常强悍。所谓的框架就是一个已经被集成了各种功能（高性能异步下载，队列，分布式，解析，持久化等）的具有很强通用性的项目模板  1.2安装1、linux 1pip3 install scrapy  2、windows 12345a. pip3 install wheelb. 下载twist">
<meta property="og:locale" content="zh-Hans">
<meta property="og:image" content="c:%5CUsers%5Cdxl%5CDesktop%5C%E5%8D%9A%E5%AE%A2%5C%E7%88%AC%E8%99%AB%5Cassets%5C1565316257620.png">
<meta property="og:image" content="c:%5CUsers%5Cdxl%5CDesktop%5C%E5%8D%9A%E5%AE%A2%5C%E7%88%AC%E8%99%AB%5Cassets%5C1565360225346.png">
<meta property="og:updated_time" content="2019-09-20T12:29:27.581Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="爬虫scripy框架">
<meta name="twitter:description" content="一介绍1.1什么是scrapy1　Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架，非常出名，非常强悍。所谓的框架就是一个已经被集成了各种功能（高性能异步下载，队列，分布式，解析，持久化等）的具有很强通用性的项目模板  1.2安装1、linux 1pip3 install scrapy  2、windows 12345a. pip3 install wheelb. 下载twist">
<meta name="twitter:image" content="c:%5CUsers%5Cdxl%5CDesktop%5C%E5%8D%9A%E5%AE%A2%5C%E7%88%AC%E8%99%AB%5Cassets%5C1565316257620.png">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/dxl/',
    scheme: 'Gemini',
    version: '5.1.4',
    sidebar: {"position":"left","display":"post","offset":12,"b2t":true,"scrollpercent":true,"onmobile":false},
    fancybox: true,
    tabs: true,
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    duoshuo: {
      userId: '0',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="http://yoursite.com/2019/09/20/【爬虫06】scrapy框架/">





  <title>爬虫scripy框架 | 我的快乐时光</title>
  








</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  
  
    
  

  <div class="container sidebar-position-left page-post-detail">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-wrapper">
  <div class="site-meta ">
    

    <div class="custom-logo-site-title">
      <a href="/dxl/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">我的快乐时光</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
      
        <p class="site-subtitle"></p>
      
  </div>

  <div class="site-nav-toggle">
    <button>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
      <span class="btn-bar"></span>
    </button>
  </div>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/dxl/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br>
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-categories">
          <a href="/dxl/categories/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-th"></i> <br>
            
            分类
          </a>
        </li>
      

      
    </ul>
  

  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal" itemscope itemtype="http://schema.org/Article">
  
  
  
  <div class="post-block">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/dxl/2019/09/20/【爬虫06】scrapy框架/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="name" content>
      <meta itemprop="description" content>
      <meta itemprop="image" content="/dxl/images/avatar.png">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="我的快乐时光">
    </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">爬虫scripy框架</h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              
                <span class="post-meta-item-text">发表于</span>
              
              <time title="创建于" itemprop="dateCreated datePublished" datetime="2019-09-20T20:44:46+08:00">
                2019-09-20
              </time>
            

            

            
          </span>

          
            <span class="post-category">
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/dxl/categories/爬虫/" itemprop="url" rel="index">
                    <span itemprop="name">爬虫</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
              <span class="post-comments-count">
                <span class="post-meta-divider">|</span>
                <span class="post-meta-item-icon">
                  <i class="fa fa-comment-o"></i>
                </span>
                <a href="/dxl/2019/09/20/【爬虫06】scrapy框架/#comments" itemprop="discussionUrl">
                  <span class="post-comments-count valine-comment-count" data-xid="/dxl/2019/09/20/【爬虫06】scrapy框架/" itemprop="commentCount"></span>
                </a>
              </span>
            
          

          
          

          
            <span class="post-meta-divider">|</span>
            <span class="page-pv">本文总阅读量
            <span class="busuanzi-value" id="busuanzi_value_page_pv"></span>次
            </span>
          

          
            <div class="post-wordcount">
              
                
                <span class="post-meta-item-icon">
                  <i class="fa fa-file-word-o"></i>
                </span>
                
                  <span class="post-meta-item-text">字数统计&#58;</span>
                
                <span title="字数统计">
                  
                </span>
              

              
                <span class="post-meta-divider">|</span>
              

              
                <span class="post-meta-item-icon">
                  <i class="fa fa-clock-o"></i>
                </span>
                
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                
                <span title="阅读时长">
                  
                </span>
              
            </div>
          

          

        </div>
      </header>
    

    
    
    
    <div class="post-body" itemprop="articleBody">

      
      

      
        <h2 id="一介绍"><a href="#一介绍" class="headerlink" title="一介绍"></a>一介绍</h2><h3 id="1-1什么是scrapy"><a href="#1-1什么是scrapy" class="headerlink" title="1.1什么是scrapy"></a>1.1什么是scrapy</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">　Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架，非常出名，非常强悍。所谓的框架就是一个已经被集成了各种功能（高性能异步下载，队列，分布式，解析，持久化等）的具有很强通用性的项目模板</span><br></pre></td></tr></table></figure>

<h3 id="1-2安装"><a href="#1-2安装" class="headerlink" title="1.2安装"></a>1.2安装</h3><p>1、linux</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">pip3 install scrapy</span><br></pre></td></tr></table></figure>

<p>2、windows</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">a. pip3 install wheel</span><br><span class="line">b. 下载twisted http://www.lfd.uci.edu/~gohlke/pythonlibs/#twisted</span><br><span class="line">c. 进入下载目录，执行 pip3 install Twisted‑17.1.0‑cp35‑cp35m‑win_amd64.whl</span><br><span class="line">d. pip3 install pywin32</span><br><span class="line">e. pip3 install scrapy</span><br></pre></td></tr></table></figure>

<h3 id="1-3基础使用"><a href="#1-3基础使用" class="headerlink" title="1.3基础使用"></a>1.3基础使用</h3><p>1、项目结构</p>
<p><img src="C:%5CUsers%5Cdxl%5CDesktop%5C%E5%8D%9A%E5%AE%A2%5C%E7%88%AC%E8%99%AB%5Cassets%5C1565316257620.png" alt="1565316257620"></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">scrapy.cfg   项目的主配置信息。（真正爬虫相关的配置信息在settings.py文件中）</span><br><span class="line">items.py     设置数据存储模板，用于结构化数据，如：Django的Model</span><br><span class="line">pipelines    数据持久化处理</span><br><span class="line">settings.py  配置文件，如：递归的层数、并发数，延迟下载等</span><br><span class="line">spiders      爬虫目录，如：创建文件，编写爬虫解析规则</span><br></pre></td></tr></table></figure>

<p>2、项目搭建及使用</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">1.</span>创建项目：scrapy startproject 项目名称</span><br><span class="line"><span class="number">2</span>、创建应用：</span><br><span class="line">	cd project_name（进入项目目录）</span><br><span class="line">	scrapy genspider 应用名称 爬取网页的起始url （scrapy genspider qiubai www.qiushibaike.com）</span><br><span class="line"><span class="number">3</span>、编写爬虫文件</span><br><span class="line">	allowed_domains = [<span class="string">'https://www.qiushibaike.com/'</span>]<span class="comment">#注释掉</span></span><br><span class="line"><span class="number">4</span>、设置修改settings.py配置文件相关配置</span><br><span class="line">	<span class="number">19</span>行：USER_AGENT = <span class="string">'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'</span> <span class="comment">#伪装请求载体身份</span></span><br><span class="line">    <span class="number">22</span>行：ROBOTSTXT_OBEY = <span class="literal">False</span>  <span class="comment">#可以忽略或者不遵守robots协</span></span><br><span class="line"><span class="number">5</span>、启动项目：</span><br><span class="line">	在项目文件夹下</span><br><span class="line">	scrapy crawl 爬虫名称 ：该种执行形式会显示执行的日志信息</span><br><span class="line">    scrapy crawl 爬虫名称 --nolog：该种执行形式不会显示行的日志信息</span><br></pre></td></tr></table></figure>

<h3 id="1-4如何提升scrapy爬取数据的效率"><a href="#1-4如何提升scrapy爬取数据的效率" class="headerlink" title="1.4如何提升scrapy爬取数据的效率"></a>1.4如何提升scrapy爬取数据的效率</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">增加并发：</span><br><span class="line">    默认scrapy开启的并发线程为32个，可以适当进行增加。在settings配置文件中修改CONCURRENT_REQUESTS = 100值为100,并发设置成了为100。</span><br><span class="line"></span><br><span class="line">降低日志级别：</span><br><span class="line">    在运行scrapy时，会有大量日志信息的输出，为了减少CPU的使用率。可以设置log输出信息为INFO或者ERROR即可。在配置文件中编写：LOG_LEVEL = ‘INFO’</span><br><span class="line"></span><br><span class="line">禁止cookie：</span><br><span class="line">    如果不是真的需要cookie，则在scrapy爬取数据时可以禁止cookie从而减少CPU的使用率，提升爬取效率。在配置文件中编写：COOKIES_ENABLED = False</span><br><span class="line"></span><br><span class="line">禁止重试：</span><br><span class="line">    对失败的HTTP进行重新请求（重试）会减慢爬取速度，因此可以禁止重试。在配置文件中编写：RETRY_ENABLED = False</span><br><span class="line"></span><br><span class="line">减少下载超时：</span><br><span class="line">    如果对一个非常慢的链接进行爬取，减少下载超时可以能让卡住的链接快速被放弃，从而提升效率。在配置文件中进行编写：DOWNLOAD_TIMEOUT = 10 超时时间为10s</span><br></pre></td></tr></table></figure>

<h3 id="1-6-Scrapy的日志等级"><a href="#1-6-Scrapy的日志等级" class="headerlink" title="1.6 Scrapy的日志等级"></a>1.6 Scrapy的日志等级</h3><p>​        在使用scrapy crawl spiderFileName运行程序时，在终端里打印输出的就是scrapy的日志信息。</p>
<p>1、日志信息的种类</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">ERROR ： 一般错误</span><br><span class="line">WARNING : 警告</span><br><span class="line">INFO : 一般的信息</span><br><span class="line">DEBUG ： 调试信息</span><br><span class="line"><span class="comment">#设置</span></span><br><span class="line">LOG_LEVEL = <span class="string">'ERROR'</span></span><br></pre></td></tr></table></figure>

<p>2、设置日志信息指定输出</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">在settings.py配置文件中，加入</span><br><span class="line">	LOG_LEVEL = ‘指定日志信息种类’即可。</span><br><span class="line">	LOG_FILE = <span class="string">'log.txt'</span>则表示将日志信息写入到指定文件中进行存储。</span><br></pre></td></tr></table></figure>

<h2 id="二、持久化存储"><a href="#二、持久化存储" class="headerlink" title="二、持久化存储"></a>二、持久化存储</h2><h3 id="2-1两种持久化持久存储方式"><a href="#2-1两种持久化持久存储方式" class="headerlink" title="2.1两种持久化持久存储方式"></a>2.1两种持久化持久存储方式</h3><p>1、基于终端指令的持久化存储</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">保证爬虫文件的parse方法中有可迭代类型对象（通常为列表or字典）的返回，该返回值可以通过终端指令的形式写入指定格式的文件中进行持久化操作。</span><br><span class="line">scrapy crawl 爬虫名称 -o xxx.json</span><br><span class="line">   scrapy crawl 爬虫名称 -o xxx.xml</span><br><span class="line">   scrapy crawl 爬虫名称 -o xxx.csv</span><br></pre></td></tr></table></figure>

<p>2、基于管道的持久化储存</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">scrapy框架中已经为我们专门集成好了高效、便捷的持久化操作功能，我们直接使用即可。要想使用scrapy的持久化操作功能，我们首先来认识如下两个文件：</span><br><span class="line">items.py：数据结构模板文件。定义数据属性。</span><br><span class="line">   pipelines.py：管道文件。接收数据（items），进行持久化操作。</span><br></pre></td></tr></table></figure>

<p>3、持久化流程</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">1.</span>爬虫文件爬取到数据后，需要将数据封装到items对象中。</span><br><span class="line"><span class="number">2.</span>使用<span class="keyword">yield</span>关键字将items对象提交给pipelines管道进行持久化操作。</span><br><span class="line"><span class="number">3.</span>在管道文件中的process_item方法中接收爬虫文件提交过来的item对象，然后编写持久化存储的代码将item对象中存储的数据进行持久化存储</span><br><span class="line"><span class="number">4.</span>settings.py配置文件中开启管道</span><br></pre></td></tr></table></figure>

<p>4、注意事项</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">爬虫类和管道类进行数据交互的形式</span><br><span class="line">- yild item：只可以交互item类型的对象</span><br><span class="line">- spider参数：可以交互任意形式的数据</span><br></pre></td></tr></table></figure>

<h2 id="三、递归解析和post数据爬取"><a href="#三、递归解析和post数据爬取" class="headerlink" title="三、递归解析和post数据爬取"></a>三、递归解析和post数据爬取</h2><h3 id="3-1知识点补充"><a href="#3-1知识点补充" class="headerlink" title="3.1知识点补充"></a>3.1知识点补充</h3><p>1、如何手动发送请求</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">yield</span> scrapy.Request(url,callback)</span><br><span class="line"><span class="keyword">yield</span> scrapy.FormRequest(url,formdata,callback)</span><br><span class="line"><span class="comment">#scrapy.FormRequest(url,formdata,callback)没有返回值</span></span><br></pre></td></tr></table></figure>

<p>2、post请求的发送和cookie的处理</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">- cookie的处理是scrapy自动封装好</span><br></pre></td></tr></table></figure>

<p>3、start_urls自动请求发送的实现原理</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">def start_requests(self):</span><br><span class="line">       for url in self.start_urls:</span><br><span class="line">           yield scrapy.Request(url=url,callback=self.parse)</span><br></pre></td></tr></table></figure>

<p>3、手动发送请求时的传参</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">请求传参：持久化存储,将不同的回调中解析的数据存储到同一个item对象。请求传参传递的就是item对象。</span><br><span class="line">使用场景：如果使用scrapy爬取的数据没有在同一张页面中</span><br><span class="line">传递方式：将传递数据封装到meta字典中，meta传递个了callback</span><br><span class="line">           yield scrapy.Request(url,callback,meta)</span><br><span class="line">接收：</span><br><span class="line">	 在callback指定的回调函数中使用response进行接收：</span><br><span class="line">  	 - item = response.meta[&apos;key&apos;]</span><br></pre></td></tr></table></figure>

<h3 id="3-2-爬取糗事百科的标题和评论"><a href="#3-2-爬取糗事百科的标题和评论" class="headerlink" title="3.2 爬取糗事百科的标题和评论"></a>3.2 爬取糗事百科的标题和评论</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line"># -*- coding: utf-8 -*-</span><br><span class="line">import scrapy</span><br><span class="line">from qiushibaike.items import QiushibaikeItem</span><br><span class="line"># scrapy.http import Request</span><br><span class="line">class QiushiSpider(scrapy.Spider):</span><br><span class="line">    name = &apos;qiushi&apos;</span><br><span class="line">    allowed_domains = [&apos;www.qiushibaike.com&apos;]</span><br><span class="line">    start_urls = [&apos;https://www.qiushibaike.com/text/&apos;]</span><br><span class="line"></span><br><span class="line">    #爬取多页</span><br><span class="line">    pageNum = 1 #起始页码</span><br><span class="line">    url = &apos;https://www.qiushibaike.com/text/page/%s/&apos; #每页的url</span><br><span class="line"></span><br><span class="line">    def parse(self, response):</span><br><span class="line">        div_list=response.xpath(&apos;//*[@id=&quot;content-left&quot;]/div&apos;)</span><br><span class="line">        for div in div_list:</span><br><span class="line">            #//*[@id=&quot;qiushi_tag_120996995&quot;]/div[1]/a[2]/h2</span><br><span class="line">            author=div.xpath(&apos;.//div[@class=&quot;author clearfix&quot;]//h2/text()&apos;).extract_first()</span><br><span class="line">            author=author.strip(&apos;\n&apos;)</span><br><span class="line">            content=div.xpath(&apos;.//div[@class=&quot;content&quot;]/span/text()&apos;).extract_first()</span><br><span class="line">            content=content.strip(&apos;\n&apos;)</span><br><span class="line">            item=QiushibaikeItem()</span><br><span class="line">            item[&apos;author&apos;]=author</span><br><span class="line">            item[&apos;content&apos;]=content</span><br><span class="line"></span><br><span class="line">            yield item #提交item到管道进行持久化</span><br><span class="line"></span><br><span class="line">         #爬取所有页码数据</span><br><span class="line">        if self.pageNum &lt;= 13: #一共爬取13页（共13页）</span><br><span class="line">            self.pageNum += 1</span><br><span class="line">            url = format(self.url % self.pageNum)</span><br><span class="line"></span><br><span class="line">            #递归爬取数据：callback参数的值为回调函数（将url请求后，得到的相应数据继续进行parse解析），递归调用parse函数</span><br><span class="line">            yield scrapy.Request(url=url,callback=self.parse)</span><br></pre></td></tr></table></figure>

<h2 id="四、scrapy中间件、UA代理、代理池"><a href="#四、scrapy中间件、UA代理、代理池" class="headerlink" title="四、scrapy中间件、UA代理、代理池"></a>四、scrapy中间件、UA代理、代理池</h2><h3 id="4-1介绍"><a href="#4-1介绍" class="headerlink" title="4.1介绍"></a>4.1介绍</h3><p>1、作用</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">批量拦截整个工程中发起的所有请求和响应</span><br></pre></td></tr></table></figure>

<p>2、为什么要拦截请求</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">- UA伪装：</span><br><span class="line">      - process_request:request.headers[&apos;User-Agent&apos;]= xxx</span><br><span class="line"> - 代理ip的设定</span><br><span class="line">      - process_exception:request.meta[&apos;proxy&apos;]= ’http://ip:port‘</span><br></pre></td></tr></table></figure>

<p>3、为什么要篡改拦截响应</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">篡改响应数据/篡改响应对象</span><br></pre></td></tr></table></figure>

<p>4、注意事项</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">中间件需要在配置文件中手动开启</span><br></pre></td></tr></table></figure>

<h3 id="4-2核心组件"><a href="#4-2核心组件" class="headerlink" title="4.2核心组件"></a>4.2核心组件</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">调度器(Scheduler)</span><br><span class="line">用来接受引擎发过来的请求, 压入队列中, 并在引擎再次请求的时候返回. 可以想像成一个URL（抓取网页的网址或者说是链接）的优先队列, 由它来决定下一个要抓取的网址是什么, 同时去除重复的网址</span><br><span class="line">下载器(Downloader)</span><br><span class="line">用于下载网页内容, 并将网页内容返回给蜘蛛(Scrapy下载器是建立在twisted这个高效的异步模型上的)</span><br><span class="line">爬虫(Spiders)</span><br><span class="line">爬虫是主要干活的, 用于从特定的网页中提取自己需要的信息, 即所谓的实体(Item)。用户也可以从中提取出链接,让Scrapy继续抓取下一个页面</span><br><span class="line">项目管道(Pipeline)</span><br><span class="line">负责处理爬虫从网页中抽取的实体，主要的功能是持久化实体、验证实体的有效性、清除不需要的信息。当页面被爬虫解析后，将被发送到项目管道，并经过几个特定的次序处理数据。</span><br></pre></td></tr></table></figure>

<h3 id="4-3框架图"><a href="#4-3框架图" class="headerlink" title="4.3框架图"></a>4.3框架图</h3><p><img src="C:%5CUsers%5Cdxl%5CDesktop%5C%E5%8D%9A%E5%AE%A2%5C%E7%88%AC%E8%99%AB%5Cassets%5C1565360225346.png" alt="1565360225346"></p>
<h3 id="4-4UA池"><a href="#4-4UA池" class="headerlink" title="4.4UA池"></a>4.4UA池</h3><p>1、作用</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">尽可能多的将scrapy工程中的请求伪装成不同类型的浏览器身份。</span><br></pre></td></tr></table></figure>

<p>2、操作流程</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">1.在下载中间件中拦截请求</span><br><span class="line">2.将拦截到的请求的请求头信息中的UA进行篡改伪装</span><br><span class="line">3.在配置文件中开启下载中间件</span><br></pre></td></tr></table></figure>

<p>3、代码展示</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">from</span> scrapy.contrib.downloadermiddleware.useragent <span class="keyword">import</span> UserAgentMiddleware</span><br><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="comment">#UA池代码的编写（单独给UA池封装一个下载中间件的一个类）</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">RandomUserAgent</span><span class="params">(UserAgentMiddleware)</span>:</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_request</span><span class="params">(self, request, spider)</span>:</span></span><br><span class="line">        <span class="comment">#从列表中随机抽选出一个ua值</span></span><br><span class="line">        ua = random.choice(user_agent_list)</span><br><span class="line">        <span class="comment">#ua值进行当前拦截到请求的ua的写入操作</span></span><br><span class="line">        request.headers.setdefault(<span class="string">'User-Agent'</span>,ua)</span><br><span class="line"></span><br><span class="line">user_agent_list = [</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span>,</span><br><span class="line">        <span class="string">"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 "</span></span><br><span class="line">        <span class="string">"(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"</span></span><br><span class="line">]</span><br></pre></td></tr></table></figure>

<h3 id="4-5代理池"><a href="#4-5代理池" class="headerlink" title="4.5代理池"></a>4.5代理池</h3><p>1、作用</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">尽可能多的将scrapy工程中的请求的IP设置成不同的</span><br></pre></td></tr></table></figure>

<p>2、操作流程</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">1.在下载中间件中拦截请求</span><br><span class="line">2.将拦截到的请求的IP修改成某一代理IP</span><br><span class="line">3.在配置文件中开启下载中间件</span><br></pre></td></tr></table></figure>

<p>3、代码展示</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line">#批量对拦截到的请求进行ip更换</span><br><span class="line">#单独封装下载中间件类</span><br><span class="line">class Proxy(object):</span><br><span class="line">    def process_request(self, request, spider):</span><br><span class="line">        #对拦截到请求的url进行判断（协议头到底是http还是https）</span><br><span class="line">        #request.url返回值：http://www.xxx.com</span><br><span class="line">        h = request.url.split(&apos;:&apos;)[0]  #请求的协议头</span><br><span class="line">        if h == &apos;https&apos;:</span><br><span class="line">            ip = random.choice(PROXY_https)</span><br><span class="line">            request.meta[&apos;proxy&apos;] = &apos;https://&apos;+ip</span><br><span class="line">        else:</span><br><span class="line">            ip = random.choice(PROXY_http)</span><br><span class="line">            request.meta[&apos;proxy&apos;] = &apos;http://&apos; + ip</span><br><span class="line"></span><br><span class="line">#可被选用的代理IP</span><br><span class="line">PROXY_http = [</span><br><span class="line">    &apos;153.180.102.104:80&apos;,</span><br><span class="line">    &apos;195.208.131.189:56055&apos;,</span><br><span class="line">]</span><br><span class="line">PROXY_https = [</span><br><span class="line">    &apos;120.83.49.90:9000&apos;,</span><br><span class="line">    &apos;95.189.112.214:35508&apos;,</span><br><span class="line">]</span><br></pre></td></tr></table></figure>

<h2 id="五、scrapy-中selenium的应用"><a href="#五、scrapy-中selenium的应用" class="headerlink" title="五、scrapy 中selenium的应用"></a>五、scrapy 中selenium的应用</h2><h3 id="5-1引言"><a href="#5-1引言" class="headerlink" title="5.1引言"></a>5.1引言</h3><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">在通过scrapy框架进行某些网站数据爬取的时候，往往会碰到页面动态数据加载的情况发生，如果直接使用scrapy对其url发请求，是绝对获取不到那部分动态加载出来的数据值。但是通过观察我们会发现，通过浏览器进行url请求发送则会加载出对应的动态加载出的数据。那么如果我们想要在scrapy也获取动态加载出的数据，则必须使用selenium创建浏览器对象，然后通过该浏览器对象进行请求发送，获取动态加载的数据值。</span><br></pre></td></tr></table></figure>

<h3 id="5-2爬取网易新闻的国内板块下的新闻数据"><a href="#5-2爬取网易新闻的国内板块下的新闻数据" class="headerlink" title="5.2爬取网易新闻的国内板块下的新闻数据"></a>5.2爬取网易新闻的国内板块下的新闻数据</h3><p> 1、需求分析</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">当点击国内超链进入国内对应的页面时，会发现当前页面展示的新闻数据是被动态加载出来的，如果直接通过程序对url进行请求，是获取不到动态加载出的新闻数据的。则就需要我们使用selenium实例化一个浏览器对象，在该对象中进行url的请求，获取动态加载的新闻数据。</span><br></pre></td></tr></table></figure>

<p>2、selenium在scrapy中使用的原理分析</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">重写爬虫文件的构造方法，在该方法中使用selenium实例化一个浏览器对象（因为浏览器对象只需要被实例化一次）</span><br><span class="line">重写爬虫文件的closed(self,spider)方法，在其内部关闭浏览器对象。该方法是在爬虫结束时被调用</span><br><span class="line">重写下载中间件的process_response方法，让该方法对响应对象进行拦截，并篡改response中存储的页面数据</span><br><span class="line">在配置文件中开启下载中间件</span><br></pre></td></tr></table></figure>

<p>3、代码展示</p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#爬虫文件</span></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">WangyiSpider</span><span class="params">(RedisSpider)</span>:</span></span><br><span class="line">    name = <span class="string">'wangyi'</span></span><br><span class="line">    <span class="comment">#allowed_domains = ['www.xxxx.com']</span></span><br><span class="line">    start_urls = [<span class="string">'https://news.163.com'</span>]</span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self)</span>:</span></span><br><span class="line">        <span class="comment">#实例化一个浏览器对象(实例化一次)</span></span><br><span class="line">        self.bro = webdriver.Chrome(executable_path=<span class="string">'/Users/bobo/Desktop/chromedriver'</span>)</span><br><span class="line"></span><br><span class="line">    <span class="comment">#必须在整个爬虫结束后，关闭浏览器</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">closed</span><span class="params">(self,spider)</span>:</span></span><br><span class="line">        print(<span class="string">'爬虫结束'</span>)</span><br><span class="line">        self.bro.quit()</span><br><span class="line"></span><br><span class="line">        </span><br><span class="line">        </span><br><span class="line"> <span class="comment">#中间件文件</span></span><br><span class="line"><span class="keyword">from</span> scrapy.http <span class="keyword">import</span> HtmlResponse    </span><br><span class="line">    <span class="comment">#参数介绍：</span></span><br><span class="line">    <span class="comment">#拦截到响应对象（下载器传递给Spider的响应对象）</span></span><br><span class="line">    <span class="comment">#request：响应对象对应的请求对象</span></span><br><span class="line">    <span class="comment">#response：拦截到的响应对象</span></span><br><span class="line">    <span class="comment">#spider：爬虫文件中对应的爬虫类的实例</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">process_response</span><span class="params">(self, request, response, spider)</span>:</span></span><br><span class="line">        <span class="comment">#响应对象中存储页面数据的篡改</span></span><br><span class="line">        <span class="keyword">if</span> request.url <span class="keyword">in</span>[<span class="string">'http://news.163.com/domestic/'</span>,<span class="string">'http://news.163.com/world/'</span>,<span class="string">'http://news.163.com/air/'</span>,<span class="string">'http://war.163.com/'</span>]:</span><br><span class="line">            spider.bro.get(url=request.url)</span><br><span class="line">            js = <span class="string">'window.scrollTo(0,document.body.scrollHeight)'</span></span><br><span class="line">            spider.bro.execute_script(js)</span><br><span class="line">            time.sleep(<span class="number">2</span>)  <span class="comment">#一定要给与浏览器一定的缓冲加载数据的时间</span></span><br><span class="line">            <span class="comment">#页面数据就是包含了动态加载出来的新闻数据对应的页面数据</span></span><br><span class="line">            page_text = spider.bro.page_source</span><br><span class="line">            <span class="comment">#篡改响应对象</span></span><br><span class="line">            <span class="keyword">return</span> HtmlResponse(url=spider.bro.current_url,body=page_text,encoding=<span class="string">'utf-8'</span>,request=request)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">return</span> response</span><br><span class="line"></span><br><span class="line"><span class="comment">#配置文件</span></span><br><span class="line">DOWNLOADER_MIDDLEWARES = &#123;</span><br><span class="line">    <span class="string">'wangyiPro.middlewares.WangyiproDownloaderMiddleware'</span>: <span class="number">543</span>,</span><br><span class="line"></span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>


      
    </div>
    
    
    

    

    

    
      <div>
        <ul class="post-copyright">
  <li class="post-copyright-author">
    <strong>本文作者：</strong>
    
  </li>
  <li class="post-copyright-link">
    <strong>本文链接：</strong>
    <a href="http://yoursite.com/2019/09/20/【爬虫06】scrapy框架/" title="爬虫scripy框架">http://yoursite.com/2019/09/20/【爬虫06】scrapy框架/</a>
  </li>
  <li class="post-copyright-license">
    <strong>版权声明： </strong>
    本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/3.0/" rel="external nofollow" target="_blank">CC BY-NC-SA 3.0</a> 许可协议。转载请注明出处！
  </li>
</ul>

      </div>
    

    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/dxl/tags/爬虫/" rel="tag"># 爬虫</a>
          
        </div>
      

      
      
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/dxl/2019/09/20/【爬虫05】移动端数据的获取/" rel="next" title="爬虫指移动端数据获取">
                <i class="fa fa-chevron-left"></i> 爬虫指移动端数据获取
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
              <a href="/dxl/2019/09/20/【爬虫07】scrapy实现分布式爬虫/" rel="prev" title="分布式爬虫">
                分布式爬虫 <i class="fa fa-chevron-right"></i>
              </a>
            
          </div>
        </div>
      

      
      
    </footer>
  </div>
  
  
  
  </article>



    <div class="post-spread">
      
    </div>
  </div>


          </div>
          


          

  
    <div class="comments" id="comments">
    </div>
  



        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap">
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview-wrap">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview-wrap sidebar-panel">
        <div class="site-overview">
          <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
            
              <img class="site-author-image" itemprop="image" src="/dxl/images/avatar.png" alt>
            
              <p class="site-author-name" itemprop="name"></p>
              <p class="site-description motion-element" itemprop="description"></p>
          </div>

          <nav class="site-state motion-element">

            
              <div class="site-state-item site-state-posts">
              
                <a href="/dxl/archives">
              
                  <span class="site-state-item-count">43</span>
                  <span class="site-state-item-name">日志</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-categories">
                <a href="/dxl/categories/index.html">
                  <span class="site-state-item-count">6</span>
                  <span class="site-state-item-name">分类</span>
                </a>
              </div>
            

            
              
              
              <div class="site-state-item site-state-tags">
                <a href="/dxl/tags/index.html">
                  <span class="site-state-item-count">6</span>
                  <span class="site-state-item-name">标签</span>
                </a>
              </div>
            

          </nav>

          

          

          
          

          
          
            <div class="links-of-blogroll motion-element links-of-blogroll-inline">
              <div class="links-of-blogroll-title">
                <i class="fa  fa-fw fa-sign-out"></i>
                我的友链
              </div>
              <ul class="links-of-blogroll-list">
                
                  <li class="links-of-blogroll-item">
                    <a href="tencent://message/?Menu=yes&uin=1258517737&Site=QQ%E6%9E%81%E5%AE%A2&Service=300&sigT=45a1e5847943b64c6ff3990f8a9e644d2b31356cb0b4ac6b24663a3c8dd0f8aa12a595b1714f9d45/" title="申请坑位" target="_blank">申请坑位</a>
                  </li>
                
              </ul>
            </div>
          

          

        </div>
      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#一介绍"><span class="nav-number">1.</span> <span class="nav-text">一介绍</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#1-1什么是scrapy"><span class="nav-number">1.1.</span> <span class="nav-text">1.1什么是scrapy</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-2安装"><span class="nav-number">1.2.</span> <span class="nav-text">1.2安装</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-3基础使用"><span class="nav-number">1.3.</span> <span class="nav-text">1.3基础使用</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-4如何提升scrapy爬取数据的效率"><span class="nav-number">1.4.</span> <span class="nav-text">1.4如何提升scrapy爬取数据的效率</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#1-6-Scrapy的日志等级"><span class="nav-number">1.5.</span> <span class="nav-text">1.6 Scrapy的日志等级</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#二、持久化存储"><span class="nav-number">2.</span> <span class="nav-text">二、持久化存储</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#2-1两种持久化持久存储方式"><span class="nav-number">2.1.</span> <span class="nav-text">2.1两种持久化持久存储方式</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#三、递归解析和post数据爬取"><span class="nav-number">3.</span> <span class="nav-text">三、递归解析和post数据爬取</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#3-1知识点补充"><span class="nav-number">3.1.</span> <span class="nav-text">3.1知识点补充</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-2-爬取糗事百科的标题和评论"><span class="nav-number">3.2.</span> <span class="nav-text">3.2 爬取糗事百科的标题和评论</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#四、scrapy中间件、UA代理、代理池"><span class="nav-number">4.</span> <span class="nav-text">四、scrapy中间件、UA代理、代理池</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#4-1介绍"><span class="nav-number">4.1.</span> <span class="nav-text">4.1介绍</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#4-2核心组件"><span class="nav-number">4.2.</span> <span class="nav-text">4.2核心组件</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#4-3框架图"><span class="nav-number">4.3.</span> <span class="nav-text">4.3框架图</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#4-4UA池"><span class="nav-number">4.4.</span> <span class="nav-text">4.4UA池</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#4-5代理池"><span class="nav-number">4.5.</span> <span class="nav-text">4.5代理池</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#五、scrapy-中selenium的应用"><span class="nav-number">5.</span> <span class="nav-text">五、scrapy 中selenium的应用</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#5-1引言"><span class="nav-number">5.1.</span> <span class="nav-text">5.1引言</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#5-2爬取网易新闻的国内板块下的新闻数据"><span class="nav-number">5.2.</span> <span class="nav-text">5.2爬取网易新闻的国内板块下的新闻数据</span></a></li></ol></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

      
        <div class="back-to-top">
          <i class="fa fa-arrow-up"></i>
          
            <span id="scrollpercent"><span>0</span>%</span>
          
        </div>
      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        <div class="copyright">&copy; <span itemprop="copyrightYear">2019</span>
  <span class="with-love">
    <i class="fa fa-hand-peace-o"></i>
  </span>
  <span class="author" itemprop="copyrightHolder"></span>

  
</div>









        
<div class="busuanzi-count">
  <script async src="https://dn-lbstatics.qbox.me/busuanzi/2.3/busuanzi.pure.mini.js"></script>

  
    <span class="site-uv">
      本站访客数
      <span class="busuanzi-value" id="busuanzi_value_site_uv"></span>
      人次
    </span>
  

  
    <span class="site-pv">
      本站总访问量
      <span class="busuanzi-value" id="busuanzi_value_site_pv"></span>
      次
    </span>
  
</div>








        
      </div>
    </footer>

    

    

  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  












  
  
    <script type="text/javascript" src="/dxl/lib/jquery/index.js?v=2.1.3"></script>
  

  
  
    <script type="text/javascript" src="/dxl/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>
  

  
  
    <script type="text/javascript" src="/dxl/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>
  

  
  
    <script type="text/javascript" src="/dxl/lib/velocity/velocity.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/dxl/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>
  

  
  
    <script type="text/javascript" src="/dxl/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>
  


  


  <script type="text/javascript" src="/dxl/js/src/utils.js?v=5.1.4"></script>

  <script type="text/javascript" src="/dxl/js/src/motion.js?v=5.1.4"></script>



  
  


  <script type="text/javascript" src="/dxl/js/src/affix.js?v=5.1.4"></script>

  <script type="text/javascript" src="/dxl/js/src/schemes/pisces.js?v=5.1.4"></script>



  
  <script type="text/javascript" src="/dxl/js/src/scrollspy.js?v=5.1.4"></script>
<script type="text/javascript" src="/dxl/js/src/post-details.js?v=5.1.4"></script>



  


  <script type="text/javascript" src="/dxl/js/src/bootstrap.js?v=5.1.4"></script>



  


  




	





  





  










  <script src="//cdn1.lncld.net/static/js/3.0.4/av-min.js"></script>
  <script src="//unpkg.com/valine/dist/Valine.min.js"></script>
  
  <script type="text/javascript">
    var GUEST = ['nick','mail','link'];
    var guest = 'nick,mail,link';
    guest = guest.split(',').filter(item=>{
      return GUEST.indexOf(item)>-1;
    });
    new Valine({
        el: '#comments' ,
        verify: false,
        notify: false,
        appId: '13B0JGDuA6ttduN8AQaR8CzF-gzGzoHsz',
        appKey: 'I13r9r5mVgq4jQYpYy6V4gW3',
        placeholder: '欢迎大佬指点~~~',
        avatar:'mm',
        guest_info:guest,
        pageSize:'10' || 10,
    });
  </script>



  





  

  

  

  
  

  

  

  

</body>
</html>
